﻿using crawler.Models;
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;

namespace crawler
{
    class Crawler
    {
        private HtmlWeb web;

        private Logger logger = Logger.getLogger();

        private string imdbURL = "http://akas.imdb.com";

        private string genericMovieURL = "http://akas.imdb.com/title/tt";

        private string genericURL = "http://akas.imdb.com/search/title?at=0&genres={0}&sort=moviemeter,asc&start={1}&title_type=feature";

        private List<string> genreList = new List<string> { "action", "adventure", "animation", "biography", "comedy", "crime", "drama", "family", "fantasy", "film_noir", "history", "horror", "music", "musical", "mystery", "romance", "sci_fi", "sport", "thriller", "war", "western" };

        private double CoveredPercentage { get; set; }
        private double StartPercentage { get; set; }

        private Database database;

        public Crawler(Database db, double percentageOfCoveredMovies = 50, double percentageStart = 0)
        {
            web = new HtmlWeb();
            database = db;
            CoveredPercentage = percentageOfCoveredMovies;
            StartPercentage = percentageStart;
        }

        public void crawlOneMovie(int imdbId)
        {
            string movieURL = genericMovieURL + imdbId;
            string imdbScoreURL = movieURL + "/ratings?ref_=tt_ov_rt";
            string metaScoreURL = movieURL + "/criticreviews?ref_=tt_ov_rt";
            
            Film film = new Film();
            List<Director> directors = new List<Director>();
            List<Writer> writers = new List<Writer>();

            film.ImdbId = imdbId;
            if (database.isMoviePresentInDatabase(film))
            {
                return;
            }

            HtmlDocument movieDoc = web.Load(movieURL);

            HtmlNode overview = getSelectedNode(movieDoc.DocumentNode, "//table[@id='title-overview-widget-layout']/tbody/tr/td[@id='overview-top']");
            if (overview != null)
            {
                film.Name = getName(overview);
                film.Duration = getDuration(overview);
                film.Year = getYear(overview);
                film.MPAARating = getMPAARating(overview);
                directors = getListOfDirectors(overview);
                writers = getListOfWriters(overview);
            }
            List<Genre> genres = getListOfGenres(movieDoc.DocumentNode);

            // visit imdbScoreURL and obtain ratings
            HtmlDocument imdbScoreDocument = web.Load(imdbScoreURL);
            fillMovieWithImdbRatings(film, imdbScoreDocument);

            // visit metaScoreURL and obtain meta rating and reviews
            HtmlDocument metaScoreDocument = web.Load(metaScoreURL);
            fillMovieWithMetaScore(film, metaScoreDocument);
            List<Review> reviews = getListOfReviews(metaScoreDocument);

            // save movie and related data to database
            database.persistMovieToDatabase(film, directors, writers, genres, reviews);

            logger.log(film.ToString());
        }

        public void doCrawling(string startingGenre = "action")
        {
            int startIndex = genreList.IndexOf(startingGenre);
            List<string> genresToProcess = genreList.GetRange(startIndex, genreList.Count - startIndex);

            foreach (string genre in genresToProcess)
            {
                string url = String.Format(genericURL, genre, "1");

                HtmlDocument genreDocument = web.Load(url);

                int numOfTitles = getNumOfTitles(genreDocument);
                int numToCover = (int)(numOfTitles * CoveredPercentage / 100);

                logger.log("Starting crawling " + genre + " genre. " + numToCover + " / " + numOfTitles + " titles will be crawled.");

                int startMovieId = (int)(numOfTitles * StartPercentage / 100) - 1;

                for (int numOfSaved = startMovieId, numOfParsed = startMovieId; numOfSaved < numToCover; )
                {
                    url = String.Format(genericURL, genre, numOfParsed + 1);
                    genreDocument = web.Load(url);

                    HtmlNodeCollection results = getSelectedNodes(genreDocument.DocumentNode, "//table[@class='results']/tr[@class]");

                    foreach (HtmlNode result in results)
                    {
                        numOfParsed++;

                        HtmlNode title = getSelectedNode(result, "td[@class='title']");
                        if (title == null)
                        {
                            continue;
                        }
                        
                        if (isReleased(title) == false)
                        {
                            continue;
                        }

                        HtmlNode href = getSelectedNode(title, "a[@href]");
                        if (href == null)
                        {
                            continue;
                        }

                        string movieURL = imdbURL + href.Attributes["href"].Value;
                        string imdbScoreURL = movieURL + "ratings?ref_=tt_ov_rt";
                        string metaScoreURL = movieURL + "criticreviews?ref_=tt_ov_rt";

                        Film film = getMovieWithIdentifiers(title, href);

                        if (database.isMoviePresentInDatabase(film))
                        {
                            numOfSaved++;
                            continue;
                        }

                        // visit movieURL and obtain directors, writers, genres and MPAA rating
                        List<Director> directors = new List<Director>();
                        List<Writer> writers = new List<Writer>();

                        HtmlDocument movieDocument = web.Load(movieURL);
                        HtmlNode overview = getSelectedNode(movieDocument.DocumentNode, "//table[@id='title-overview-widget-layout']/tbody/tr/td[@id='overview-top']");
                        if (overview != null)
                        {
                            film.Year = getYear(overview);
                            film.MPAARating = getMPAARating(overview);
                            directors = getListOfDirectors(overview);
                            writers = getListOfWriters(overview);
                        }
                        List<Genre> genres = getListOfGenres(movieDocument.DocumentNode);


                        // visit imdbScoreURL and obtain ratings
                        HtmlDocument imdbScoreDocument = web.Load(imdbScoreURL);
                        fillMovieWithImdbRatings(film, imdbScoreDocument);


                        // visit metaScoreURL and obtain meta rating and reviews
                        HtmlDocument metaScoreDocument = web.Load(metaScoreURL);
                        fillMovieWithMetaScore(film, metaScoreDocument);
                        List<Review> reviews = getListOfReviews(metaScoreDocument);


                        // save movie and related data to database
                        database.persistMovieToDatabase(film, directors, writers, genres, reviews);


                        numOfSaved++;
                        logger.log(numOfSaved + ". " + film.ToString());

                        if (numOfSaved >= numToCover)
                        {
                            break;
                        }

                    }
                }
            }
        }

        private List<Review> getListOfReviews(HtmlDocument metaScoreDocument)
        {
            List<Review> reviews = new List<Review>();

            HtmlNodeCollection tableRows = getSelectedNodes(metaScoreDocument.DocumentNode, "//table[@class='crits_results']/tbody/tr[@itemprop='reviews']");

            foreach (HtmlNode row in tableRows)
            {
                HtmlNode review = getSelectedNode(row, "td[@class='review']/div[@class='summary'][@itemprop='reviewbody']");
                reviews.Add(new Review(HtmlEntity.DeEntitize(review.InnerText.Trim())));
            }

            return reviews;
        }

        private void fillMovieWithMetaScore(Film film, HtmlDocument metaScoreDocument)
        {
            HtmlNode metaScoreBlock = getSelectedNode(metaScoreDocument.DocumentNode, "//div[@class='metascore_block'][@itemprop='aggregateRating']");

            if (metaScoreBlock != null)
            {
                film.MetaScore = getMetaScore(metaScoreBlock);
                film.NumOfMetaScoreVoters = getNumOfMetaScoreVoters(metaScoreBlock);
            }
        }

        private int? getNumOfMetaScoreVoters(HtmlNode metaScoreBlock)
        {
            HtmlNode numOfVoters = getSelectedNode(metaScoreBlock, "span[@itemprop='ratingCount']");
            if (numOfVoters != null)
            {
                return stringToInt(numOfVoters.InnerText);
            }
            return null;
        }

        private int? getMetaScore(HtmlNode metaScoreBlock)
        {
            HtmlNode metaScore = getSelectedNode(metaScoreBlock, "div[@class='metascore_wrap']/div[@class]/span[@itemprop='ratingValue']");
            if (metaScore != null)
            {
                return stringToInt(metaScore.InnerText);
            }
            return null;
        }

        private void fillMovieWithImdbRatings(Film film, HtmlDocument scoreDocument)
        {
            HtmlNode content = getSelectedNode(scoreDocument.DocumentNode, "//div[@id='tn15content']");
            if (content == null)
            {
                return;
            }

            HtmlNode paragraph = getSelectedNode(content, "p");
            if (paragraph != null)
            {
                Match match = Regex.Match(paragraph.InnerText, @"(\d+) IMDb users have given .* of (\d.?\d) / 10");

                if (match.Groups.Count == 3)
                {
                    film.NumOfImdbScoreVoters = stringToInt(match.Groups[1].Value);
                    film.ImdbScore = Convert.ToDouble(match.Groups[2].Value.Replace('.', ','));
                }
            }


            HtmlNode table = getSelectedNode(content, "table[2]");
            if (table == null)
            {
                return;
            }

            HtmlNode males = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-male']");
            if (males != null)
            {
                HtmlNode malesScore = getSelectedNode(males.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (malesScore != null)
                {
                    film.ImdbMaleScore = Convert.ToDouble(HtmlEntity.DeEntitize(malesScore.InnerText).Trim().Replace('.', ','));
                }
            }

            HtmlNode females = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-female']");
            if (females != null)
            {
                HtmlNode femalesScore = getSelectedNode(females.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (femalesScore != null)
                {
                    film.ImdbFemaleScore = Convert.ToDouble(HtmlEntity.DeEntitize(femalesScore.InnerText).Trim().Replace('.', ','));
                }
            }

            HtmlNode ageBelow18 = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-age_1']");
            if (ageBelow18 != null)
            {
                HtmlNode ageBelow18Score = getSelectedNode(ageBelow18.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (ageBelow18Score != null)
                {
                    film.ImdbBelow18Score = Convert.ToDouble(HtmlEntity.DeEntitize(ageBelow18Score.InnerText).Trim().Replace('.', ','));
                }
            }

            HtmlNode age18_29 = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-age_2']");
            if (age18_29 != null)
            {
                HtmlNode age18_29Score = getSelectedNode(age18_29.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (age18_29Score != null)
                {
                    film.Imdb18_29Score = Convert.ToDouble(HtmlEntity.DeEntitize(age18_29Score.InnerText).Trim().Replace('.', ','));
                }
            }

            HtmlNode age30_44 = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-age_3']");
            if (age30_44 != null)
            {
                HtmlNode age30_44Score = getSelectedNode(age30_44.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (age30_44Score != null)
                {
                    film.Imdb30_44Score = Convert.ToDouble(HtmlEntity.DeEntitize(age30_44Score.InnerText).Trim().Replace('.', ','));
                }
            }

            HtmlNode ageAbove45 = getSelectedNode(table, "tr/td[@align='right'][@nowrap='1']/a[@href='ratings-age_4']");
            if (ageAbove45 != null)
            {
                HtmlNode ageAbove45Score = getSelectedNode(ageAbove45.ParentNode.ParentNode, "td[@background][@nowrap]");
                if (ageAbove45Score != null)
                {
                    film.ImdbAbove45Score = Convert.ToDouble(HtmlEntity.DeEntitize(ageAbove45Score.InnerText).Trim().Replace('.', ','));
                }
            }
        }

        private Film getMovieWithIdentifiers(HtmlNode titleNode, HtmlNode href)
        {
            Film film = new Film();

            film.Name = HtmlEntity.DeEntitize(href.InnerText); ;
            film.ImdbId = stringToInt(Regex.Match(href.Attributes["href"].Value, @"\d+").Value);

            HtmlNode durationNode = getSelectedNode(titleNode, "span[@class='runtime']");
            if (durationNode != null)
            {
                film.Duration = stringToInt(Regex.Match(durationNode.InnerText, @"\d+").Value);
            }

            return film;
        }

        private List<Genre> getListOfGenres(HtmlNode documentNode)
        {
            List<Genre> genres = new List<Genre>();

            HtmlNodeCollection genresNode = getSelectedNodes(documentNode, "//div[@id='maindetails_center_bottom']/div[@id='titleStoryLine']/div[@itemprop='genre']/a");

            foreach (HtmlNode genre in genresNode)
            {
                genres.Add(new Genre(genre.InnerText.Trim()));
            }

            return genres;
        }

        private int? getYear(HtmlNode overviewNode)
        {
            HtmlNode year = getSelectedNode(overviewNode, "h1[@class='header']/span[@class='nobr']/a[@href]");
            if (year != null)
            {
                return stringToInt(year.InnerText);
            }
            return null;
        }

        private string getName(HtmlNode overviewNode)
        {
            HtmlNode name = getSelectedNode(overviewNode, "h1[@class='header']/span[@class='itemprop'][@itemprop='name']");
            if (name != null)
            {
                return HtmlEntity.DeEntitize(name.InnerText);
            }
            return null;
        }

        private int? getDuration(HtmlNode overviewNode)
        {
            HtmlNode duration = getSelectedNode(overviewNode, "div[@class='infobar']/time[@itemprop='duration']");
            if (duration != null)
            {
                return stringToInt(Regex.Match(duration.InnerText, @"\d+").Value);
            }
            return null;
        }

        private string getMPAARating(HtmlNode overviewNode)
        {
            HtmlNode mpaa = getSelectedNode(overviewNode, "div[@class='infobar']/span[@itemprop='contentRating']");
            if (mpaa != null)
            {
                return mpaa.Attributes["content"].Value;
            }
            return null;
        }

        private List<Director> getListOfDirectors(HtmlNode overviewNode)
        {
            List<Director> directors = new List<Director>();

            HtmlNodeCollection directorsNode = getSelectedNodes(overviewNode, "div[@itemprop='director']/a[@itemprop='url']");

            foreach (HtmlNode director in directorsNode)
            {
                HtmlNode name = getSelectedNode(director, "span[@itemprop='name']");
                if (name != null)
                {
                    directors.Add(new Director(HtmlEntity.DeEntitize(name.InnerText)));
                }
            }

            return directors;
        }

        private List<Writer> getListOfWriters(HtmlNode overviewNode)
        {
            List<Writer> writers = new List<Writer>();

            HtmlNodeCollection writersNode = getSelectedNodes(overviewNode, "div[@itemprop='creator']/a[@itemprop='url']");

            foreach (HtmlNode writer in writersNode)
            {
                HtmlNode name = getSelectedNode(writer, "span[@itemprop='name']");
                if (name != null)
                {
                    writers.Add(new Writer(HtmlEntity.DeEntitize(name.InnerText)));
                }
            }

            return writers;
        }

        private int? stringToInt(string str)
        {
            int integer;
            if (Int32.TryParse(str, out integer))
            {
                return integer;
            }
            return null;
        }

        private HtmlNode getSelectedNode(HtmlNode currentNode, string xpath)
        {
            HtmlNodeCollection nodes = currentNode.SelectNodes(xpath);
            return nodes == null ? null : nodes[0];
        }

        private HtmlNodeCollection getSelectedNodes(HtmlNode currentNode, string xpath)
        {
            HtmlNodeCollection nodes = currentNode.SelectNodes(xpath);
            return nodes == null ? new HtmlNodeCollection(null) : nodes;
        }

        private bool isReleased(HtmlNode titleNode)
        {
            HtmlNodeCollection rating = titleNode.SelectNodes("div[@class='user_rating']/div[@class='rating rating-list']/div[@class='rating-ineligible']");
            return rating == null;
        }

        private int getNumOfTitles(HtmlDocument document)
        {
            HtmlNode node = getSelectedNode(document.DocumentNode, "//div[@class='leftright']/div[@id='left']");
            if (node == null)
            {
                return 0;
            }

            Match match = Regex.Match(node.InnerText, @"of (\d+),?(\d+)");

            if (match.Groups.Count < 3)
            {
                return 0;
            }
            int numOfMovies = stringToInt(match.Groups[1].Value + match.Groups[2].Value) ?? 0;

            return numOfMovies;
        }

    }
}
